evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/api/benchmark/__init__.py

@@ -1,3 +1,11 @@
-from .adapters import DefaultDataAdapter, MultiChoiceAdapter, Text2ImageAdapter
+from .adapters import (
+    AgentAdapter,
+    DefaultDataAdapter,
+    ImageEditAdapter,
+    MultiChoiceAdapter,
+    NERAdapter,
+    Text2ImageAdapter,
+    VisionLanguageAdapter,
+)
 from .benchmark import DataAdapter
 from .meta import BenchmarkMeta

evalscope/api/benchmark/adapters/__init__.py

@@ -1,3 +1,7 @@
+from .agent_adapter import AgentAdapter
 from .default_data_adapter import DefaultDataAdapter
+from .image_edit_adapter import ImageEditAdapter
 from .multi_choice_adapter import MultiChoiceAdapter
+from .ner_adapter import NERAdapter
 from .text2image_adapter import Text2ImageAdapter
+from .vision_language_adapter import VisionLanguageAdapter

evalscope/api/benchmark/adapters/agent_adapter.py

@@ -0,0 +1,8 @@
+from .default_data_adapter import DefaultDataAdapter
+
+
+class AgentAdapter(DefaultDataAdapter):
+    """Adapter for agent benchmarks. e.g., function calling, etc."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)

evalscope/api/benchmark/adapters/default_data_adapter.py

@@ -2,6 +2,7 @@ import os
 from collections import defaultdict
 from functools import partial
 from overrides import override
+from tqdm.auto import tqdm
 from typing import Any, Callable, Dict, List, Optional, Tuple, Type
 
 from evalscope.api.dataset import DataLoader, Dataset, DatasetDict, LocalDataLoader, RemoteDataLoader, Sample
@@ -128,6 +129,9 @@ class DefaultDataAdapter(DataAdapter):
             for sample in self.test_dataset[subset]:
                 if isinstance(sample.input, str):
                     sample.input = self.process_sample_str_input(sample, subset)
+                elif isinstance(sample.input, list):
+                    # Handle list[ChatMessage] and add system prompt if needed
+                    sample.input = self.process_sample_messages_input(sample, subset)
 
     def process_sample_str_input(self, sample: Sample, subset: str) -> List[ChatMessage]:
         """
@@ -142,6 +146,15 @@ class DefaultDataAdapter(DataAdapter):
             input_messages.insert(0, ChatMessageSystem(content=self.system_prompt))
         return input_messages
 
+    def process_sample_messages_input(self, sample: Sample, subset: str) -> List[ChatMessage]:
+        """
+        Normalize a sample's existing List[ChatMessage] input and ensure system prompt is set once.
+        """
+        messages = list(sample.input)  # shallow copy to avoid in-place mutations
+        if self.system_prompt and not any(isinstance(m, ChatMessageSystem) for m in messages):
+            messages = [ChatMessageSystem(content=self.system_prompt)] + messages
+        return messages
+
     def process_sample_input(self, sample: Sample, subset: str) -> str:
         """
         Process a single sample's input by applying prompt templates and few-shot formatting.
@@ -241,6 +254,7 @@ class DefaultDataAdapter(DataAdapter):
             filter_func=self.sample_filter,
             limit=self.limit if not self.reformat_subset else None,  # Limit number of samples if specified
             repeats=self.repeats,  # Number of repetitions for each sample
+            shuffle=self.shuffle,  # Shuffle dataset if enabled
             shuffle_choices=self.shuffle_choices,  # Shuffle choices if requested
             data_source=self.dataset_hub,  # Data source configuration
         )
@@ -599,6 +613,61 @@ class DefaultDataAdapter(DataAdapter):
 
         return sample_score
 
+    def batch_match_score(
+        self, original_predictions: List[str], filtered_predictions: List[str], references: List[str],
+        task_states: List[TaskState]
+    ) -> Optional[List[Score]]:
+        """
+        Batch calculate evaluation scores by comparing predictions with references.
+
+        This method computes scores using all configured metrics for a batch of samples
+        and creates a list of Score objects with detailed evaluation results.
+
+        Args:
+            original_predictions (List[str]): The original, unfiltered model predictions
+            filtered_predictions (List[str]): The filtered and processed predictions
+            references (List[str]): The ground truth reference answers
+            task_states (List[TaskState]): The complete task states for context
+
+        Returns:
+            List[Score]: List of objects containing all calculated metric scores and metadata
+        """
+        return None  # Default implementation does not support batch scoring
+
+    @override
+    def batch_calculate_metrics(self, task_states: List[TaskState],
+                                sample_scores: List[SampleScore]) -> List[SampleScore]:
+        """Batch calculate metrics for a list of task states with tqdm progress and batch processing."""
+        total = len(task_states)
+        if total == 0:
+            return sample_scores
+
+        # Prepare lists for batch processing
+        original_predictions: List[str] = []
+        filtered_predictions: List[str] = []
+        references: List[str] = []
+
+        for ts in task_states:
+            pred = ts.output.completion
+            original_predictions.append(pred)
+            filtered_predictions.append(self.filter_prediction(pred, ts))
+            references.append(ts.target)
+
+        batch_scores = self.batch_match_score(
+            original_predictions=original_predictions,
+            filtered_predictions=filtered_predictions,
+            references=references,
+            task_states=task_states
+        )
+
+        if batch_scores is not None:
+            assert len(batch_scores) == len(sample_scores), \
+                'Batch scores length must match sample scores length.'
+            for batch_score, sample_score in zip(batch_scores, sample_scores):
+                sample_score.score.value.update(batch_score.value)
+
+        return sample_scores
+
     @override
     def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
         """
@@ -641,9 +710,7 @@ class DefaultDataAdapter(DataAdapter):
         """
         pass
 
-    def _on_generate_report(
-        self, scores: Dict[str, List[AggScore]], model_name: str, add_aggregation_name: bool = True
-    ) -> Report:
+    def _on_generate_report(self, scores: Dict[str, List[AggScore]], model_name: str) -> Report:
         """
         Hook method called during report generation.
 
@@ -659,7 +726,7 @@ class DefaultDataAdapter(DataAdapter):
             Report: The generated evaluation report
         """
         return ReportGenerator.generate_report(
-            score_dict=scores, model_name=model_name, data_adapter=self, add_aggregation_name=add_aggregation_name
+            score_dict=scores, model_name=model_name, data_adapter=self, add_aggregation_name=self.add_aggregation_name
         )
 
     @override
@@ -681,3 +748,7 @@ class DefaultDataAdapter(DataAdapter):
         report = self._on_generate_report(scores, model_name=model_name)
         self._on_generate_report_end(report, output_dir, **kwargs)
         return report
+
+    def finalize(self, *args, **kwargs):
+        # Finalize the evaluation process
+        self.sandbox_finalize(*args, **kwargs)

evalscope/api/benchmark/adapters/image_edit_adapter.py

@@ -0,0 +1,82 @@
+import os
+from typing import Optional
+
+from evalscope.constants import EvalType, FileConstants
+from evalscope.utils import get_logger
+from evalscope.utils.function_utils import thread_safe
+from evalscope.utils.io_utils import jsonl_to_list
+from .text2image_adapter import Text2ImageAdapter
+
+logger = get_logger()
+
+
+class ImageEditAdapter(Text2ImageAdapter):
+    """
+    Support two methods:
+    1. Inference using modelscope pipeline
+    2. Load local inference jsonl file with key to corresponding prompt
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.local_file = self.extra_params.get('local_file', None)
+        self.id_key = self.extra_params.get('id_key', FileConstants.ID)
+        self.image_key = self.extra_params.get('image_key', FileConstants.IMAGE_PATH)
+        self.local_data = self.load_local_file()
+
+    def load_local_file(self) -> Optional[dict]:
+        if not self.local_file:
+            return None
+
+        # Load file and check
+        data_list = jsonl_to_list(self.local_file)
+        data_dict = {}
+        for record in data_list:
+            if self.image_key not in record:
+                raise ValueError(f"Image key '{self.image_key}' not found in record: {record}, file {self.local_file}")
+            if self.id_key not in record:
+                raise ValueError(f"ID key '{self.id_key}' not found in record: {record}, file {self.local_file}")
+
+            image_path = record[self.image_key]
+            if not os.path.isabs(image_path):
+                image_path = os.path.join(os.path.dirname(self.local_file), image_path)
+            if not os.path.exists(image_path):
+                raise FileNotFoundError(f"Image file '{image_path}' not found.")
+
+            data_dict[record[self.id_key]] = record
+        return data_dict
+
+    def get_image_path_from_id(self, image_id) -> Optional[str]:
+        if not self.local_file:
+            return None
+
+        record = self.local_data.get(image_id)
+        if not record:
+            return None
+
+        return record[self.image_key]
+
+    def _post_process_samples(self):
+        super()._post_process_samples()
+
+        # Add local image path if exists
+        for subset in self.test_dataset.keys():
+            for sample in self.test_dataset[subset]:
+                local_image_path = self.get_image_path_from_id(sample.metadata.get(FileConstants.ID))
+                if local_image_path:
+                    sample.metadata[FileConstants.IMAGE_PATH] = local_image_path
+
+    def sample_filter(self, sample) -> bool:
+        """
+        Filter samples based on metadata availability.
+        If local file is not available, all samples are considered valid.
+        Otherwise, only samples with valid metadata and image path are kept.
+        """
+        if not self.local_data:
+            return True
+        else:
+            sample_id = sample.metadata.get(FileConstants.ID)
+            if (not sample_id) or (not self.get_image_path_from_id(sample_id)):
+                return False
+            return True

evalscope/api/benchmark/adapters/multi_choice_adapter.py

@@ -18,8 +18,11 @@ class MultiChoiceAdapter(DefaultDataAdapter):
     This adapter formats the input for multi-choice questions and handles few-shot examples.
     """
 
-    multiple_correct: bool = False
-    """Whether the benchmark allows multiple correct answers."""
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.multiple_correct: bool = False
+        """Whether the benchmark allows multiple correct answers."""
 
     def format_prompt_template(self, sample: Sample) -> str:
         """

evalscope/api/benchmark/adapters/ner_adapter.py

@@ -0,0 +1,212 @@
+from typing import Any, Dict, List, Set, Tuple
+
+from evalscope.api.dataset import Sample
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.utils.import_utils import check_import
+from evalscope.utils.logger import get_logger
+from evalscope.utils.ner import (
+    DEFAULT_TAG_FIX_PATTERNS,
+    calculate_bio_metrics,
+    clean_prediction,
+    create_target_text,
+    extract_entities_from_text,
+    extract_spans_from_bio,
+    xml_to_bio_tags,
+)
+from .default_data_adapter import DefaultDataAdapter
+
+logger = get_logger()
+
+
+class NERAdapter(DefaultDataAdapter):
+    """
+    Base adapter class for Named Entity Recognition (NER) tasks.
+
+    This adapter handles converting between BIO tagging schemes and XML-style entity markup,
+    and provides evaluation metrics using seqeval.
+
+    Subclasses should define their entity types and register the benchmark.
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        # Define mapping from BIO tags to user-friendly tag names
+        self.entity_type_map = {}
+        # Add descriptions for each entity type
+        self.entity_descriptions = {}
+
+        # These will be initialized in setup_entity_mappings
+        self.reverse_entity_map = {}
+        self.entity_list = []
+        self.entities_description = ''
+
+        # Define common error patterns to handle
+        self.tag_fix_patterns = DEFAULT_TAG_FIX_PATTERNS
+
+        check_import('seqeval', 'seqeval', raise_error=True, feature_name='NER metrics')
+        # Note: setup_entity_mappings() should be called by subclasses
+        # after they define their entity_type_map and entity_descriptions
+
+    def setup_entity_mappings(self):
+        """
+        Setup entity mappings and descriptions for prompt formatting.
+        This should be called after entity_type_map and entity_descriptions are defined.
+        """
+        # Reverse mapping for converting back from prediction to evaluation
+        self.reverse_entity_map = {v.lower(): k for k, v in self.entity_type_map.items()}
+
+        # Create list of tags for prompt formatting
+        self.entity_list = [f'<{ent.lower()}>' for ent in self.entity_type_map.values()]
+
+        # Create description of entities for prompt
+        self.entities_description = ', '.join([
+            f'{self.entity_type_map[tag]} ({self.entity_descriptions[tag]})' for tag in self.entity_type_map
+        ])
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a record with tokens and NER tags into a Sample.
+        Creates both the raw text input and annotated text target.
+        """
+        tokens: List[str] = record['tokens']
+        ner_tags: List[str] = record['ner_tags']
+
+        # Create the input text by joining tokens
+        input_text = ' '.join(tokens)
+
+        # Process tokens and tags to create annotated target text
+        target_text = create_target_text(tokens, ner_tags, self.entity_type_map)
+
+        # Store tokens and tags in metadata for evaluation
+        metadata = {'tokens': tokens, 'ner_tags': ner_tags}
+
+        return Sample(input=input_text, target=target_text, metadata=metadata)
+
+    def format_prompt_template(self, sample):
+        """
+        Format the prompt with entity types, available tags, and text to annotate.
+        """
+        return self.prompt_template.format(
+            entities=self.entities_description, entity_list=', '.join(self.entity_list), text=sample.input
+        )
+
+    def format_fewshot_template(self, fewshot, sample):
+        """
+        Format the few-shot prompt with all required parameters.
+        """
+        return self.few_shot_prompt_template.format(
+            fewshot=fewshot,
+            entities=self.entities_description,
+            entity_list=', '.join(self.entity_list),
+            text=sample.input
+        )
+
+    def sample_to_fewshot(self, sample: Sample) -> str:
+        """
+        Format a sample as a few-shot example showing original and annotated text.
+        """
+        if not sample.metadata:
+            return ''
+
+        # Format few-shot examples to match the expected response format
+        return f'Input:\n{sample.input}\n\nOutput:\n{sample.target}'
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        """
+        Evaluate named entity recognition performance using seqeval.
+        """
+        from seqeval.metrics import accuracy_score, f1_score, precision_score, recall_score
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        try:
+            # Get the original tokens and tags from the reference metadata
+            original_tokens = task_state.metadata['tokens']
+            original_tags = task_state.metadata['ner_tags']
+
+            if not original_tokens or len(original_tokens) == 0:
+                if hasattr(reference, 'metadata') and reference.metadata:
+                    original_tokens = reference.metadata['tokens']
+                    original_tags = reference.metadata['ner_tags']
+
+            # Clean and normalize the prediction
+            cleaned_prediction = clean_prediction(filtered_prediction, self.tag_fix_patterns)
+
+            # Convert XML-style prediction back to BIO tags aligned with original tokens
+            pred_bio_tags = xml_to_bio_tags(cleaned_prediction, original_tokens, self.reverse_entity_map)
+
+            # Use seqeval to calculate metrics
+            # Note: seqeval expects lists of lists (one per sequence)
+            y_true = [original_tags]
+            y_pred = [pred_bio_tags]
+
+            precision = precision_score(y_true, y_pred)
+            recall = recall_score(y_true, y_pred)
+            f1 = f1_score(y_true, y_pred)
+            accuracy = accuracy_score(y_true, y_pred)
+
+            score.value = {'precision': precision, 'recall': recall, 'f1_score': f1, 'accuracy': accuracy}
+
+            # Store tags for aggregation (proper micro-averaging in aggregate_scores)
+            # This way aggregate_scores can compute metrics across all samples at once,
+            # which gives you true micro-averaged scores rather than averaged macro scores.
+            score.metadata = {'y_true': original_tags, 'y_pred': pred_bio_tags}
+        except Exception as e:
+            logger.warning(f'Error evaluating NER prediction: {str(e)}')
+            score.value = {'precision': 0.0, 'recall': 0.0, 'f1_score': 0.0, 'accuracy': 0.0}
+
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Aggregate metrics across all samples using seqeval.
+        """
+        from seqeval.metrics import accuracy_score, f1_score, precision_score, recall_score
+
+        # Collect all predictions and references
+        y_true_all = []
+        y_pred_all = []
+
+        for ss in sample_scores:
+            # Extract the BIO tags from metadata if available
+            # You may need to store these during match_score
+            if hasattr(ss.score, 'metadata') and 'y_true' in ss.score.metadata and 'y_pred' in ss.score.metadata:
+                y_true_all.append(ss.score.metadata['y_true'])
+                y_pred_all.append(ss.score.metadata['y_pred'])
+
+        if not y_true_all:
+            # Fallback: calculate averages from individual scores
+            num_samples = len(sample_scores)
+            avg_precision = sum(ss.score.value.get('precision', 0.0) for ss in sample_scores) / num_samples
+            avg_recall = sum(ss.score.value.get('recall', 0.0) for ss in sample_scores) / num_samples
+            avg_f1 = sum(ss.score.value.get('f1_score', 0.0) for ss in sample_scores) / num_samples
+            avg_accuracy = sum(ss.score.value.get('accuracy', 0.0) for ss in sample_scores) / num_samples
+        else:
+            # Use seqeval for micro-averaged metrics across all samples
+            avg_precision = precision_score(y_true_all, y_pred_all)
+            avg_recall = recall_score(y_true_all, y_pred_all)
+            avg_f1 = f1_score(y_true_all, y_pred_all)
+            avg_accuracy = accuracy_score(y_true_all, y_pred_all)
+
+        num_samples = len(sample_scores)
+
+        agg_scores = [
+            AggScore(
+                metric_name='precision',
+                score=avg_precision,
+                num=num_samples,
+                metadata={'type': 'seqeval-micro-average'}
+            ),
+            AggScore(
+                metric_name='recall', score=avg_recall, num=num_samples, metadata={'type': 'seqeval-micro-average'}
+            ),
+            AggScore(metric_name='f1_score', score=avg_f1, num=num_samples, metadata={'type': 'seqeval-micro-average'}),
+            AggScore(
+                metric_name='accuracy', score=avg_accuracy, num=num_samples, metadata={'type': 'seqeval-accuracy'}
+            )
+        ]
+
+        return agg_scores

evalscope/api/benchmark/adapters/text2image_adapter.py

@@ -8,7 +8,7 @@ from evalscope.api.messages.content import ContentImage
 from evalscope.api.metric import Score
 from evalscope.api.model import ChatCompletionChoice, Model, ModelOutput
 from evalscope.api.registry import get_metric
-from evalscope.constants import EvalType
+from evalscope.constants import EvalType, FileConstants
 from evalscope.utils import get_logger
 from evalscope.utils.function_utils import thread_safe
 from .default_data_adapter import DefaultDataAdapter
@@ -19,6 +19,11 @@ logger = get_logger()
 class Text2ImageAdapter(DefaultDataAdapter):
     """Text to Image Adapter for benchmarks."""
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.add_aggregation_name = False  # Do not add aggregation name in the report by default
+
     def load_from_disk(self, **kwargs):
         return super().load_from_disk(use_local_loader=True)
 
@@ -27,11 +32,12 @@ class Text2ImageAdapter(DefaultDataAdapter):
         return Sample(
             input=[ChatMessageUser(content=record['prompt'])],
             metadata={
-                'id': record['id'],
                 'prompt': record['prompt'],
                 'category': record.get('category', ''),
                 'tags': record.get('tags', []),
-                'image_path': record.get('image_path', ''),  # Optional field for existing image path
+                FileConstants.ID: record[FileConstants.ID],
+                FileConstants.IMAGE_PATH: record.get(FileConstants.IMAGE_PATH,
+                                                     ''),  # Optional field for existing image path
             }
         )
 
@@ -83,7 +89,7 @@ class Text2ImageAdapter(DefaultDataAdapter):
                 completed=True,
             )
         else:
-            image_id = f"{sample.metadata.get('id',sample.id)}_{sample.group_id}"
+            image_id = f'{sample.metadata.get(FileConstants.ID, sample.id)}_{sample.group_id}'
             output_path = os.path.join(output_dir, 'images', f'{image_id}.png')
             if not os.path.exists(os.path.dirname(output_path)):
                 os.makedirs(os.path.dirname(output_path))
@@ -96,7 +102,7 @@ class Text2ImageAdapter(DefaultDataAdapter):
             with open(output_path, 'wb') as f:
                 f.write(base64.b64decode(image_base64))
 
-            sample.metadata['image_path'] = output_path
+            sample.metadata[FileConstants.IMAGE_PATH] = output_path
             return TaskState(
                 model=model.name,
                 sample=sample,
@@ -111,7 +117,7 @@ class Text2ImageAdapter(DefaultDataAdapter):
         self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
     ) -> Score:
         # Get prediction and prompt from task state
-        image_path = task_state.metadata.get('image_path', original_prediction)
+        image_path = task_state.metadata.get(FileConstants.IMAGE_PATH, original_prediction)
         prompt = task_state.input[0].content
         meta = task_state.metadata
 
@@ -149,7 +155,3 @@ class Text2ImageAdapter(DefaultDataAdapter):
                 score.metadata[metric_name] = f'error: {str(e)}'
 
         return score
-
-    def _on_generate_report(self, scores, model_name, add_aggregation_name=True):
-        # Don't add aggregation name for needle haystack adapter
-        return super()._on_generate_report(scores, model_name, False)

evalscope/api/benchmark/adapters/vision_language_adapter.py

@@ -0,0 +1,8 @@
+from .default_data_adapter import DefaultDataAdapter
+
+
+class VisionLanguageAdapter(DefaultDataAdapter):
+    """Adapter for vision-language benchmarks. e.g., image captioning, visual question answering, etc."""
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)